{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Otto商品分类——RBF 核SVM\n",
    "\n",
    "我们以Kaggle 2015年举办的Otto Group Product Classification Challenge竞赛数据为例，分别调用\n",
    "缺省参数SVC、\n",
    "SVC + GridSearchCV进行参数调优。\n",
    "\n",
    "Otto数据集是著名电商Otto提供的一个多类商品分类问题，类别数=9. 每个样本有93维数值型特征（整数，表示某种事件发生的次数，已经进行过脱敏处理）。 竞赛官网：https://www.kaggle.com/c/otto-group-product-classification-challenge/data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#竞赛的评价指标为logloss\n",
    "#from sklearn.metrics import log_loss  \n",
    "\n",
    "#SVM虽然也支持输出各类的概率，但这需要额外的计算费用，且得到的概率也不保证是合法的概率，\n",
    "#所以在这个例子中我们用正确率accuracy_score作为模型选择的度量，最后在最佳超参数情况下再训练模型，得到概率表示\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>feat_1_tfidf</th>\n",
       "      <th>feat_2_tfidf</th>\n",
       "      <th>feat_3_tfidf</th>\n",
       "      <th>feat_4_tfidf</th>\n",
       "      <th>feat_5_tfidf</th>\n",
       "      <th>feat_6_tfidf</th>\n",
       "      <th>feat_7_tfidf</th>\n",
       "      <th>feat_8_tfidf</th>\n",
       "      <th>feat_9_tfidf</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_85_tfidf</th>\n",
       "      <th>feat_86_tfidf</th>\n",
       "      <th>feat_87_tfidf</th>\n",
       "      <th>feat_88_tfidf</th>\n",
       "      <th>feat_89_tfidf</th>\n",
       "      <th>feat_90_tfidf</th>\n",
       "      <th>feat_91_tfidf</th>\n",
       "      <th>feat_92_tfidf</th>\n",
       "      <th>feat_93_tfidf</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.081393</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.075886</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.231403</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.199730</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.011987</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011668</td>\n",
       "      <td>0.105971</td>\n",
       "      <td>0.021681</td>\n",
       "      <td>0.080435</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.008244</td>\n",
       "      <td>0.022456</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.124622</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.145988</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 95 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  feat_1_tfidf  feat_2_tfidf  feat_3_tfidf  feat_4_tfidf  feat_5_tfidf  \\\n",
       "0   1      0.081393           0.0           0.0      0.000000      0.000000   \n",
       "1   2      0.000000           0.0           0.0      0.000000      0.000000   \n",
       "2   3      0.000000           0.0           0.0      0.000000      0.000000   \n",
       "3   4      0.011987           0.0           0.0      0.011668      0.105971   \n",
       "4   5      0.000000           0.0           0.0      0.000000      0.000000   \n",
       "\n",
       "   feat_6_tfidf  feat_7_tfidf  feat_8_tfidf  feat_9_tfidf   ...     \\\n",
       "0      0.000000      0.000000      0.000000           0.0   ...      \n",
       "1      0.000000      0.000000      0.231403           0.0   ...      \n",
       "2      0.000000      0.000000      0.199730           0.0   ...      \n",
       "3      0.021681      0.080435      0.000000           0.0   ...      \n",
       "4      0.000000      0.000000      0.000000           0.0   ...      \n",
       "\n",
       "   feat_85_tfidf  feat_86_tfidf  feat_87_tfidf  feat_88_tfidf  feat_89_tfidf  \\\n",
       "0       0.075886       0.000000       0.000000            0.0            0.0   \n",
       "1       0.000000       0.000000       0.000000            0.0            0.0   \n",
       "2       0.000000       0.000000       0.000000            0.0            0.0   \n",
       "3       0.000000       0.008244       0.022456            0.0            0.0   \n",
       "4       0.124622       0.000000       0.000000            0.0            0.0   \n",
       "\n",
       "   feat_90_tfidf  feat_91_tfidf  feat_92_tfidf  feat_93_tfidf   target  \n",
       "0       0.000000            0.0            0.0            0.0  Class_1  \n",
       "1       0.000000            0.0            0.0            0.0  Class_1  \n",
       "2       0.000000            0.0            0.0            0.0  Class_1  \n",
       "3       0.000000            0.0            0.0            0.0  Class_1  \n",
       "4       0.145988            0.0            0.0            0.0  Class_1  \n",
       "\n",
       "[5 rows x 95 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "\n",
    "#原始特征 + tf_idf特征对线性SVM训练还是很快，RBF核已慢得不行\n",
    "# RBF核只用tf_idf特征\n",
    "train = pd.read_csv(dpath +\"Otto_FE_train_tfidf.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 将类别字符串变成数字\n",
    "# drop ids and get labels\n",
    "y_train = train['target']   #形式为Class_x\n",
    "X_train = train.drop([\"id\", \"target\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "\n",
    "#sklearn的学习器大多之一稀疏数据输入，模型训练会快很多\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/qing/anaconda2/lib/python2.7/site-packages/sklearn/model_selection/_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# 训练样本6w+，交叉验证太慢，用train_test_split估计模型性能\n",
    "# SVM对大样本数据集支持不太好\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train_part, X_val, y_train_part, y_val = train_test_split(X_train, y_train, train_size = 0.8,random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(49502, 93)\n"
     ]
    }
   ],
   "source": [
    "print (X_train_part.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RBF核SVM正则参数调优\n",
    "\n",
    "RBF核是SVM最常用的核函数。\n",
    "RBF核SVM 的需要调整正则超参数包括C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和核函数的宽度gamma\n",
    "C越小，决策边界越平滑； \n",
    "gamma越小，决策边界越平滑。\n",
    "\n",
    "采用交叉验证，网格搜索步骤与Logistic回归正则参数处理类似，在此略。\n",
    "\n",
    "这里我们用校验集（X_val、y_val）来估计模型性能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fit_grid_point_RBF(C, gamma, X_train, y_train, X_val, y_val):\n",
    "    \n",
    "    # 在训练集是那个利用SVC训练\n",
    "    SVC3 =  SVC( C = C, kernel='rbf', gamma = gamma)\n",
    "    SVC3 = SVC3.fit(X_train, y_train)\n",
    "    \n",
    "    # 在校验集上返回accuracy\n",
    "    accuracy = SVC3.score(X_val, y_val)\n",
    "    \n",
    "    print(\"C= {} and gamma = {}: accuracy= {} \" .format(C, gamma, accuracy))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "accuracy_s = np.matrix(np.zeros(shape=(5, 3)), float)\n",
    "gamma_s = np.logspace(-1, 1, 3)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 0.1 and gamma = 0.1: accuracy= 0.738122171946 \n",
      "C= 0.1 and gamma = 1.0: accuracy= 0.763897866839 \n",
      "C= 0.1 and gamma = 10.0: accuracy= 0.549208144796 \n"
     ]
    }
   ],
   "source": [
    "oneC = 0.1\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[0,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 1 and gamma = 0.1: accuracy= 0.761716224952 \n",
      "C= 1 and gamma = 1.0: accuracy= 0.800985778927 \n",
      "C= 1 and gamma = 10.0: accuracy= 0.71630575307 \n"
     ]
    }
   ],
   "source": [
    "oneC = 1\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[1,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 10 and gamma = 0.1: accuracy= 0.78603749192 \n",
      "C= 10 and gamma = 1.0: accuracy= 0.817065287654 \n",
      "C= 10 and gamma = 10.0: accuracy= 0.727052359405 \n"
     ]
    }
   ],
   "source": [
    "oneC = 10\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[2,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 100 and gamma = 0.1: accuracy= 0.805914673562 \n",
      "C= 100 and gamma = 1.0: accuracy= 0.807449903038 \n",
      "C= 100 and gamma = 10.0: accuracy= 0.724951519069 \n"
     ]
    }
   ],
   "source": [
    "oneC = 100\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[3,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 1000 and gamma = 0.1: accuracy= 0.805914673562 \n",
      "C= 1000 and gamma = 1.0: accuracy= 0.794521654816 \n",
      "C= 1000 and gamma = 10.0: accuracy= 0.724062702004 \n"
     ]
    }
   ],
   "source": [
    "oneC = 1000\n",
    "\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    accuracy_s[4,j] = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#需要调优的参数\n",
    "#C_s = np.logspace(-1, 3, 5)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份 \n",
    "#gamma_s = np.logspace(-1, 1, 3)    \n",
    "\n",
    "#accuracy_s = []\n",
    "#for i, oneC in enumerate(C_s):\n",
    "    #for j, gamma in enumerate(gamma_s):\n",
    "        #tmp = fit_grid_point_RBF(oneC, gamma, X_train_part, y_train_part, X_val, y_val)\n",
    "        #accuracy_s[i,j] = tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从上述结果会发现，gamma参数非常重要(当gamma=0.01或gamma=100时性能很差),非线性模型比线性模型性能好（注意我们这里只用了tfidf特征）。\n",
    "但速度慢了不是一点半点(sklearn建议核方法SVM样本数不超过10000)\n",
    "可以考虑将训练样本分为多个子集，每个子集训练一个RBF核SVM模型，最后多个模型融合的结果的到最终模型（训练速度加快，但测试可能更慢）\n",
    "\n",
    "C= 0.1 and gamma = 0.1: accuracy= 0.738122171946 \n",
    "C= 0.1 and gamma = 1.0: accuracy= 0.763897866839 \n",
    "C= 0.1 and gamma = 10.0: accuracy= 0.549208144796 \n",
    "\n",
    "C= 1 and gamma = 0.1: accuracy= 0.761716224952 \n",
    "C= 1 and gamma = 1.0: accuracy= 0.800985778927 \n",
    "C= 1 and gamma = 10.0: accuracy= 0.71630575307\n",
    "\n",
    "C= 10 and gamma = 0.1: accuracy= 0.78603749192 \n",
    "C= 10 and gamma = 1.0: accuracy= 0.817065287654 \n",
    "C= 10 and gamma = 10.0: accuracy= 0.727052359405 \n",
    "\n",
    "C= 100 and gamma = 0.1: accuracy= 0.805914673562 \n",
    "#### C= 100 and gamma = 1.0: accuracy= 0.807449903038 \n",
    "C= 100 and gamma = 10.0: accuracy= 0.724951519069\n",
    "\n",
    "C= 1000 and gamma = 0.1: accuracy= 0.805914673562 \n",
    "C= 1000 and gamma = 1.0: accuracy= 0.794521654816 \n",
    "C= 1000 and gamma = 10.0: accuracy= 0.724062702004 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VNX9+P/XO/u+kQWSEMISWRIh\nQFgErbijomg/tGhr3bePy8el9au2tP0UsWJbbP1Va7WWutSKrSv1Y2txq20VJCAii8oiSwgYICEJ\nCVnn/P64N5PJxkyWm5vl/Xw85pGZO+fOvGeSzHvOPee8rxhjUEoppY4nyO0AlFJK9X2aLJRSSvml\nyUIppZRfmiyUUkr5pclCKaWUX5oslFJK+aXJQimllF+aLJRSSvmlyUIppZRfIW4H0FOSk5NNdna2\n22EopVS/sm7dukPGmBR/7QZMssjOzqawsNDtMJRSql8Rkd2BtNPDUEoppfzSZKGUUsovTRZKKaX8\n0mShlFLKL00WSiml/NJkoZRSyi9NFkoppfwaMOsslOpTGuuhbDcc+gIObwMEYtIgJtX+mQaRiRCk\n39dU/6DJQqnuOHYEDm+3ksKhL+DQNutn6U7wNBx/36AQiE6FmJS2icT3enQKhMeCSO+8JqXaoclC\nKX88Hijfa/UQmpJB08+jXzW3CwqBpNGQfAKMm2f9TD4BkscAAkdLrPZHv/K57rPtwKfWbdPYNoaQ\nSJ8E0jqp+GyPToXQiF57a9TgoclCqSZ11T69BJ+kcHg7NBxrbheRYCWBMWdBck5zUkgcAcGhHT9+\nRJydOI7D44FjpR0kFfv64R2w+wOrXbvPE98ymUSntt9riU6GoODOv09qUNJkoQYXY6wPXO9hI5/k\nUL7Hp6FYH/7JJ8CoU1smhaghzh0SCgqyPsSjkyEt9/htG+qg6qD1epp+tk4wxR9bP+uOtt1fgiAq\nuW3vpE1SSbHGV/Qw2KCmyUINTA11UPZl27GEQ9ugtqK5XWi09W0/awYkf6c5KSSN7vuHc0LCID7D\nuvhTV9Wyd9I6qVSVWO/N0a+gsa7t/sFhrXooqe0fBotJg7Donn+tynWaLFT/Vl3qkwi+aD6MVPpl\ny2P/selWIpi40O4h2EkhLn1wfGMOi4akkdbleIyBmiPtjKn4JJryIti3zurNYNp5rpi24yhtEozd\nYwkJc+Tlqp6nyUL1fZ5GOLKnZVJoul59qLldcBgMGQOpE2DCRT5JIceaTaT8E7EOOUUmQsrY47dt\nbIDqw/ZhsJL2E0zJVjj6HtSUt/8YkUmteid2MonLgIQs6xKdqlOM+wBNFqrvqD3aasZR0wDzDmis\nbW4XNcSecXSez4yjHEgYoQO2vSk4BGLTrIs/9TX2uEp7h8HsMZeitZjKrxDfyQSAJyiMmugM6xKT\nSU20fYnJoCY6k/qI5MHROzyO2IgQpmQlOvocmixU7zIGKorbTwoV+5rbSRAkjrRnHZ3RnBSG5ED0\nEPfiV10TGgEJw62LraKmni3FFWzaV86W4go2V1SwvbqSSE81Q6WUTDlIphwiUw4yvL6EzPJiMuUT\n0qWyxUPXmFD2mWSKTIrPJZm9JpUik8Ih4oCBnUzyhyfw6s2zHX0OR5OFiMwFHgaCgSeNMUtb3Z8F\nPA0k2G3uMca8Yd93L3AN0Aj8jzHmTSdjVT2svsZamNZicNkeU/CdmRMWa/UKsk9pOeMoaSSEhLsX\nv+pRBytr2VRsJ4Xicjbtq2BPabX3/tTYcHLT4zg7N43c9DhSYttOLmgAdgF76qsIO7qPsKNFhB8t\nIuxoEUlHixhauZfZR9cTUlvWYj9PcDi1MZnU2Zem67Wx1s+GAdAziQ53vkftWLIQkWDgUeAsoAhY\nKyIrjTFbfJotAv5sjHlMRCYAbwDZ9vVLgFwgHXhLRE4wpr3VSso1xljHrFvPNjr0BRzZDcbT3DZ+\nuJUMsi6zxhWakkLs0H7/j6qaGWMoKjvG5uJyNhdXsNnuOZRUNh9GzEqKIi8jjoXThjMhPY7c9DhS\n20kOHUsEMoEZ7d9dWwlH9lrjXEf2EHRkN5FH9hB5ZA/s+bTt+pSQyObxkRaXEfaYSf9PJj3ByZ7F\ndGC7MWYngIisAOYDvsnCAHH29Xig2L4+H1hhjKkFvhSR7fbjfehgvMofjwf2roZNL8OBjVZSOObz\nLS4kwkoE6fkw8ZvNYwlDxuh0ygGo0WPYefConRSs3sKW/RWUH6sHIEggJzWWk8ckMyE9jryMeCak\nxxEXcZyFiz0hPBbSJliX9tRWehNJ82W39XNfYcu/aWhOJokj2k8oTq676UOcTBYZwF6f20W0/Srw\nv8A/RORWIBo402ff1a32DWAyuXLEwc9h4wuw8S/WwrXQKEif0nbGUfxwHWAeoGobGvniwFErKdi9\nhs/2V3Ks3ursh4UEMX5oLOedOIxcOzGMGxpLRGgf/HsIj7UWPHa06LGmwirvcmSPVQzSN5ns/cia\nWuwrNKptb6RFMkkaEMnEyWTR3rvTelL2pcBTxphlInIS8KyI5AW4LyJyPXA9QFZWVjfDVS1UHoBN\nL1lJYv8n1oDzqNPg9EUw7nwIj3E7QuWQo7UNbN1vHT5qOpS07atKGjzWv2BMeAgT0uO4ZPpw8tLj\nyc2IY3RKDKHBA2R6a0QcRBwvmZS3OMzVnEx2w941bacJh0a37ZF4eykj+s3qeCeTRREw3Od2Js2H\nmZpcA8wFMMZ8KCIRQHKA+2KMeQJ4AqCgoKCd1UGqU2qPwmevWwli53vWmMOwfDjnAcj7r8CmSKp+\npbSqznsIabM9AP3l4SqM/d80JDqM3Ix45oxNsRJDehxZSVEEBfX9DzfHRMTD0HgYmtf+/ceONPdM\nWieUPauhtlUyCYvpYMykbyUTJ5PFWiBHREYC+7AGrL/Vqs0e4AzgKREZD0QAB4GVwJ9E5CGsAe4c\n4CMHYx28GhusxLDxBStR1FdDfBacfKc17uBvYZbqF4wx7C+vadFb2Fxczv7yGm+bjIRIctPjuGhy\nBrnpceSmx5MWF470gQ+qfiUywboMPbH9+48daSeR2JfdH7QsRwPWjMEOk0lWryUTx5KFMaZBRG4B\n3sSaFrvcGLNZRBYDhcaYlcB3gd+JyB1Yh5muNMYYYLOI/BlrMLwBuFlnQvUgY6wCcxv/DJtetBZE\nRSRYpTAmLoThM3TFbD/m8Ri+PFzlTQib7V5DWbU18CwCo5KjmT4yyZsUctPjSIjS0hu9oimZDJvY\n/v3HjjSPkbS+7Po31LVcZ0J4HIz8GlzynKNhizED4+hNQUGBKSwsdDuMvq1slzVIvfEFa1FccBic\nMNdKEDln6bqGfqiuwcO2kkorMdi9hq37K6iqs75bhQYLJ6TFescWctPjGT8slqgwXY/bLzXV7vJN\nIGW7rUH0Ofd06SFFZJ0xpsBfO/2LGeiqS2HzK1YvYq89wWzEbJh1C0yYb3VhVb9QXdfA1v2VbGka\nY9hfzhcHjlLXaK1niQoLZsKwOBZMzbR6Cxlx5KTGEhaivcQBw7d217BJvfrUmiwGovoa2PYmfPIC\nbPsHeOoheSyc8SM48RvWcU7Vp5VX17eYprq5uIKdB49iT0giMSqU3PR4rpqdTW6GdRgpe0g0wYN5\n4Fk5SpPFQOHxwJ4PrENMm1+zZlzEpMGMG6yB6qET+8SMCtWSMYaSylqfgWer17DvSHMxvWHxEeSm\nx3G+vYYhNyOe9PgIHXhWvUqTRX9XsrV5wVxFkTWne/wFMGkhjDxVF8n1IR6PYU9pdXNSKK5gS3E5\nh442n2xoZHI0+VkJXDZzhD34HMeQGB1LUu7TZNEfVey3ZjFtfAEOfAoSDKNPhzP/1yrbraU1+oTD\nR2tZu6uMwl2lbNxXztbiCiprGwAICRLGpMYwZ2yqd0bS+GGxxDpdCkOpLtJk0V/UVsJWe8Hcl/+0\nFsylT4G5D0Le160TxijXGGP1GtbuKmPtl6Ws3V3KzoNVgFUKY8KwOOZPTic3PZ689Hhy0mL6ZikM\npTqgyaIva6yHHe/aC+b+DxqOWSs6T/meXagvx+0IB61Gj2Hr/grW7iqlcFcZa3eVeiurxkWEMC07\niW9MHc607EROzIwnPEQTg+rfNFn0NcbAvvVWgtj0knXa0IgEyL8UJl4Cw6frQLULauob+XjPEQp3\nlfLRrlI+3nOEo/YhpYyESE4aPYSC7CSmZyeRkxozuMthqAFJk0VfUbqzecFc6Q4IDoex9oK5MWfp\nie17WVlVHYW7y7zJYdO+cuobrXmrY9NiuWhyOtOykyjITiIjIdLlaJVyniYLN1WX2pVd/wxFdumr\n7FPg5Nth/IVWSQDluKYT9qzdVeodkN5WYp3NLyw4iImZ8Vxz8iimZScydUSilsVQg5Imi95Wfwy+\n+LuVILb9AzwNkDLemsmUt6DFOYqVMxo9hs8PVFK4u5SPvrTGHA5UWAX1YsNDmJqdyEWTM5iWncTE\nzHgdiFYKTRa9w+OB3f+2DjFtWWlVlYwdBjP/2zrMlJan4xAOqqlvZGNRud1zKGXd7jIqa6zxhqFx\nEUwbmcS07EQKRiQxdmisroJWqh2aLJz01WYrQXz6IlTss+rWj7/Qmsk08mu6YM4h5dX1rNtTykdf\n2mscisq99ZNyUmOYNzGdadmJTMtOIjMxUldCKxUATRY9raIYPv2LdZjpq03WgrkxZ8JZi2HseRAW\n5XaEA07xkWPeXsPaL8v4/CurhHNIkHBiZjxXzs5mWnYSU0ckkhSt4w1KdYUmi55QUwFb/2ovmHsf\nMJBRAOf+HHIvhpgUtyMcMDwew7aSo97kULirzFtHKSY8hCkjEpk3cRgF2UnkD08gMkx7b0r1BE0W\nXdVYD9vfthLE529AQw0kjoRT77YOMw0Z7XaEA0JtQyOb9pV7DykV7i6j/Jh1Ep+U2HCmZydx7Skj\nmZadxLihsYQMlPNAK9XHaLLoDGOgqNCu7PoyVB+GyCSYfJk1UJ05TQequ6mipp519vqGtV+WsaHo\nCHUN1njDqJRo5uYO9Q5IZyVF6XiDUr1Ek0UgDu+wxyFesBbPhUTA2HOtBDH6DF0w1w0Hymuaxxt2\nlfHZgQqMgeAgIS8jnstnjqAgO4mC7ESStfqqUq7RZNGRqkP2GeZegKK1gMDIU+CU71olwCPi3Y6w\n3zHGsOPgUe8hpbW7S9lbao03RIUFMyUrkdvOyGF6dhL5WQl66k+l+hD9b/RVf8waf9j4Z9j+lrVg\nLjXXmsmUtwDiM9yOsF+pa/CwqbjcSgz2yuiyamu8ITkmjIIRSVw5ayTTshOZMCxOxxuU6sM0WXga\nYde/rASxZSXUVUJsOsy8yTrMNDTP7Qj7jaO1Daz3qae0Ye8Rauqt8YaRydGcOT6NadlJTBuZRPYQ\nHW9Qqj/RZFG+F56ZD2GxMGG+NZMp+2RdMBeAkooa6/wNu0op3F3KluIKPAaCBHLT47l0ehbTs5OY\nmp1IamyE2+EqpbpBk0ViNlz2MoyYBaFaPbQjHo813mBVYi2jcHcpuw9XAxARGsTk4YnccnoO07IT\nmZyVSEy4/mkpNZDofzTAmDPcjqDPqa5r4JO95azbbdVSWr/niHd9Q1J0GAUjErlsxgimjUwiNz2O\nUB1vUGpA02ShAGsKa6GdGNbtLmNLcQUNHuv8DTmpMZybN5SpIxIpyNbxBqUGI00Wg1BDo4fPDlR6\nE8O63c0lMyJCg8gfnsANp46iYEQSk7MS9PwNSilNFoNBRU09H+85YieGUjbsOUJVXSMAaXHhFIxI\n4pqTRzJ1RCIT9JCSUqodmiwGGGMMe0uPsW6PVWRv3W6rCquxZymNGxrHf03NZOoI66xvGQlaolsp\n5Z+jyUJE5gIPA8HAk8aYpa3u/yVwmn0zCkg1xiTY9zUCn9r37THGXOhkrP1VXYOHzcXldj2lMtbt\nKeNgZS1gVWGdnJXAuXnDmDoikfysBJ2lpJTqEsc+OUQkGHgUOAsoAtaKyEpjzJamNsaYO3za3wpM\n9nmIY8aYfKfi669Kq+qshW+7y1i/u4xPio5QaxfaG54UycljkpkyIpGCEYmckKZnfVNK9Qwnv2ZO\nB7YbY3YCiMgKYD6wpYP2lwI/djCefseqpVTlnb5auLuMnQerAAgNFnLT47ls5ggK7ENKqXG68E0p\n5Qwnk0UGsNfndhEwo72GIjICGAm847M5QkQKgQZgqTHmVacC7Stq6hv5ZO8R1u0pY519SOmIXUsp\nMSqUqSMSWTA1k4IRSUzMjCciVFeZK6V6h5PJor3jH6aDtpcALxpjGn22ZRljikVkFPCOiHxqjNnR\n4glErgeuB8jKyuqJmHtVSUUNhfbU1cLdZWzeV+5d2zA6JZqzJ6RRMCKJKSMSGZ0SrQPRSinXOJks\nioDhPrczgeIO2l4C3Oy7wRhTbP/cKSLvYY1n7GjV5gngCYCCgoKOElGf0OgxfH6g0u41lLJuT5m3\nPHd4SBCTMhO47mujKBhhlcvQc0UrpfoSJ5PFWiBHREYC+7ASwrdaNxKRsUAi8KHPtkSg2hhTKyLJ\nwGzgZw7G2uOO1jbw8Z7mRW8f7znC0doGwDodaMGIRK44KZupIxLJTY8nLETXNiil+i7HkoUxpkFE\nbgHexJo6u9wYs1lEFgOFxpiVdtNLgRXGGN+ewXjgcRHxAEFYYxYdDYy7zhjDviPHmqev7rbO+OYx\n1llWx6bFctHkdKtcxogkMhN1bYNSqn+Rlp/R/VdBQYEpLCzsleeqb/SwpbjCO321cHcpX1VYaxui\nw4KZnJXoXfQ2OSuB2IjQXolLKaU6S0TWGWMK/LXTFVoBOFJdx3r7kFLhLmttQ9NJfTISIpk5aog3\nOYxNi9UzvimlBhxNFq0YY/jyUJVPr6GM7SVHAQgJEnLT47h0ehYFI5KYOiKRofG6tkEpNfAN+mRR\n1+Dhk6Ij3l7D+j1llFbVARAfGcqUrAQunpzB1BGJTMpMIDJM1zYopQafQZ8sSqvq+MZvrYlYo5Kj\nOX1cqndF9OiUGIK0XIZSSmmyGBofwR+umsbEjHiGxIS7HY5SSvVJgz5ZAJw2NtXtEJRSqk/TaTtK\nKaX80mShlFLKL00WSiml/NJkoZRSyi8d4FbKYdX11dR76gkNCiUkKISQoBCCRL+nqf5Fk4VSXVTv\nqefwscOUVJdQUl3CV9VfcbD6oPd2yTHrZ1V9VZt9gySoRfIIEeun77am6y3atb4t7bdtvU+7jyOh\nhAaHeh/D336tf2rCG1w0WSjVijGGirqK5g/91hc7CRw+dhjT6nxeIUEhpESmkBqVypiEMcxKn0VK\nZArhweE0eBqo99Q3/zQN1vXG5uu+bXwv9Z56ahpqvPvVN9ptOtivscV5xJwRJEHNyaqDpNNe8muR\n5NrZLzQolOCgYAQhSIIIkiBEhGAJtq77bPfeH+g2EYII6tH9m7YFS7B13Wf/jl5HR9v6cjVqTRZq\nUKltrKWkusTbA/DtDXxV/RUHj1nXaxtr2+ybEJ5AalQqqVGpjE8aT0qUlRTSotK8CSIxIrFPfOP2\nGE+7SaRFIjINbRPYcZKV7371jfUtEl57+7V32zfhtZcom9o1mkaMMXiMp01CHsiakkh7yfF428Yn\njefh0x92NDZNFmpA8BgPpTWl3kTwVfVX1vVjPterD3Kk9kibfcODw71JIC85j9RI63pqdKr3ekqU\n1TvoL4IkiLDgMMKC+/8ZF5uShgdP8/WmS3vb7ATTeltX9m+dtFpva9q/xTa7bbvbPI0tYmva1t3X\nlhmb6fjvQZOF6vOq66u9PQDfJODbMzh47CANnoYW+wlCcmQyKVEpZMRkMCV1ircHkBaV5u0ZxIXF\n9enu/2DX9O05GC3i6SZNFso1DZ4GDh075Lc3cLT+aJt9Y0JjvN/4pw2dZl2PTCEtKs27PTkymZAg\n/RNXqifof5LqcccdID7WfL3dAWIJISUqhZSolBYDxL69gbSoNKJCo1x6dUoNTposVKeV15az48gO\n64O/qm0SOFh9kJrGmjb7NQ0Qp0SlMC5pXLu9gaSIpD4xQKyUakmTheqUA1UH+K+V/0VFXYV3W9MA\ncUpkCnlD8kgdnuodMG669LcBYqVUS5osVKcs/WgpdY11PHzawwyPHa4DxEoNEposVMDe2/seb+95\nm9un3M7pWae7HY5SqhfpwWEVkOr6ah5Y8wBjEsZwee7lboejlOpl2rNQAXl84+MUVxXz1NynCA0K\ndTscpVQvC6hnISIvicj5IjpNZTDaVraNZzY/w8VjLmZq2lS3w1FKuSDQD//HgG8B20RkqYiMczAm\n1Yd4jIclq5cQExbDHVPvcDscpZRLAkoWxpi3jDHfBqYAu4BVIvKBiFwlInpMYgB7bftrrC9Zz51T\n7yQxItHtcJRSLgn4sJKIDAGuBK4FPgYexkoeqxyJTLmurKaMZeuWMSV1CvPHzHc7HKWUiwIds3gZ\n+BcQBVxgjLnQGPOCMeZWIOY4+80Vkc9FZLuI3NPO/b8UkQ325QsROeJz3xUiss2+XNH5l6a665fr\nfklVXRWLZi7SVdVKDXKBzoZ6xBjzTnt3GGMK2tsuIsHAo8BZQBGwVkRWGmO2+Ox7h0/7W4HJ9vUk\n4MdAAWCAdfa+ZQHGq7pp3VfreGX7K1yddzU5iTluh6OUclmgXxfHi0hC0w0RSRSRm/zsMx3YbozZ\naYypA1YAxzuWcSnwvH39HGCVMabUThCrgLkBxqq6qb6xniWrl5Aenc4NE29wOxylVB8QaLK4zhjj\nPURkf4Bf52efDGCvz+0ie1sbIjICGAk09V4C3lf1vGe2PMP2I9v5/ozva3VXpRQQeLIIEp/iP/Yh\nJn+n4GqvWFBH50e8BHjRGO+JgwPaV0SuF5FCESk8ePCgn3BUIPYd3cdvP/ktZ2SdwanDT3U7HKVU\nHxFosngT+LOInCEip2MdLvq7n32KgOE+tzOB4g7aXkLzIaiA9zXGPGGMKTDGFKSkpPgJR/ljjOGn\na36KiHDP9DbzEZRSg1igyeJurENE/w3cDLwN/D8/+6wFckRkpIiEYSWEla0bichYIBH40Gfzm8DZ\n9thIInC2vU056J097/B+0fvcnH8zQ6OHuh2OUqoPCWg2lDHGg7WK+7FAH9gY0yAit2B9yAcDy40x\nm0VkMVBojGlKHJcCK4wxxmffUhG5DyvhACw2xpQG+tyq86rqq3jgowcYmziWb4//ttvhKKX6mICS\nhYjkAA8AE4CIpu3GmFHH288Y8wbwRqttP2p1+3872Hc5sDyQ+FT3/WbDbyipLmHZnGV63mqlVBuB\nHob6A1avogE4DXgGeNapoFTv+qz0M57b+hwLTljApJRJboejlOqDAk0WkcaYtwExxuy2ewN69psB\nwGM83PfhfcSHx3PblNvcDkcp1UcFeryhxi5Pvs0eh9gHpDoXluotL37xIhsPbeSnJ/+U+PB4t8NR\nSvVRgfYsbseqC/U/wFTgMkDrNfVzh44d4lfrf8X0odOZN2qe2+Eopfowvz0LewHeN40xdwFHgasc\nj0r1imWFy6hpqGHRzEX4rLlUSqk2/PYs7FXVU0U/TQaUNfvX8PrO17k672pGxo90OxylVB8X6JjF\nx8BrIvIXoKppozHmZUeiUo6qa6xjyeolDI8dzrUnXut2OEqpfiDQZJEEHKblDCgDaLLoh5ZvWs6u\nil389szfEhES4X8HpdSgF+gKbh2nGCD2VOzhdxt/x9zsuczOmO12OEqpfiLQFdx/oJ2qr8aYq3s8\nIuUYYwz3r7mfsOAw7pp2l9vhKKX6kUAPQ73ucz0CuJiOK8iqPurNXW/yQfEH3Dv9XlKjdJmMUipw\ngR6Gesn3tog8D7zlSETKEZV1lTy49kEmDJnAwrEL3Q5HKdXPdLViXA6Q1ZOBKGf9+uNfc/jYYR45\n/RGCg4LdDkcp1c8EOmZRScsxiwNY57hQ/cDmQ5tZ8dkKLhl3CbnJuW6Ho5TqhwI9DBXrdCDKGY2e\nRn7y4U8YEjmEWyff6nY4Sql+KqDaUCJysYjE+9xOEJGLnAtL9ZQVn69ga+lW7p52N7FhmvOVUl0T\naCHBHxtjyptuGGOOAD92JiTVU0qqS/j1x79mVvoszsk+x+1wlFL9WKDJor12ejq1Pu7na39OfWM9\nP5jxAy0UqJTqlkCTRaGIPCQio0VklIj8EljnZGCqe/6z7z/8fdffuW7idWTF6cQ1pVT3BJosbgXq\ngBeAPwPHgJudCkp1T01DDfevuZ/suGyuztNF9kqp7gt0NlQVcI/Dsage8uSnT7K3ci9Pnv0kYcFh\nboejlBoAAp0NtUpEEnxuJ4rIm86FpbpqZ/lOfr/p98wbNY8Zw2a4HY5SaoAI9DBUsj0DCgBjTBl6\nDu4+xxjD/avvJzIkku8WfNftcJRSA0igycIjIt5RUhHJpp0qtMpdr+98nY8OfMTtU24nOTLZ7XCU\nUgNIoNNffwD8W0T+ad/+GnC9MyGpriivLecXhb9gYspEFpywwO1wlFIDTKAD3H8XkQKsBLEBeA1r\nRpTqIx5e/zDlteU8cdYTBEmgHUallApMoIUErwVuAzKxksVM4ENanmZVuWRDyQb+8sVfuHzC5YxN\nGut2OEqpASjQr6C3AdOA3caY04DJwEHHolIBa/A0cN/q+0iLSuOm/JvcDkcpNUAFmixqjDE1ACIS\nboz5DPD7FVZE5orI5yKyXUTaXachIt8UkS0isllE/uSzvVFENtiXlQHGOeg8t/U5vij7gnun30t0\naLTb4SilBqhAB7iL7HUWrwKrRKQMP6dVFZFg4FHgLKAIWCsiK40xW3za5AD3ArONMWUi4jsd95gx\nJr8Tr2XQOVB1gEc3PMqpmadyepYeEVRKOSfQAe6L7av/KyLvAvHA3/3sNh3YbozZCSAiK4D5wBaf\nNtcBj9rrNjDGlHQi9kFv6UdLMcZw74x7tVCgUspRnZ42Y4z5pzFmpTGmzk/TDGCvz+0ie5uvE4AT\nROQ/IrJaROb63BchIoX2dj13Rivv7X2Pt/e8zY2TbiQjpvXbqpRSPcvJMuPtfdVtvZAvBOt83nOw\nZlr9S0Ty7NXiWcaYYhEZBbwjIp8aY3a0eAKR67HXe2RlDZ7KqtX11Tyw5gHGJIzh8tzL3Q5HKTUI\nODkhvwgY7nM7k7bjHEXAa8YQRkzkAAAbnUlEQVSYemPMl8DnWMkDY0yx/XMn8B7WDKwWjDFPGGMK\njDEFKSkpPf8K+qjHNz5OcVUxi2YuIjQo1O1wlFKDgJPJYi2QIyIjRSQMuARoPavpVeA0ABFJxjos\ntdMuVBjus302Lcc6Bq1tZdt4ZvMzXDzmYqamTXU7HKXUIOHYYShjTIOI3AK8CQQDy40xm0VkMVBo\njFlp33e2iGwBGoG7jDGHRWQW8LiIeLAS2lLfWVSDlcd4WLJ6CTFhMdwx9Q63w1FKDSKOnhrVGPMG\n8EarbT/yuW6AO+2Lb5sPgBOdjK0/em37a6wvWc/iWYtJjEh0Oxyl1CCiRYT6ibKaMpatW8aU1CnM\nHzPf7XCUUoOMJot+4qF1D1FVV8WimYu0UKBSqtfpp04/UHigkFe3v8rluZeTk5jjdjhKqUFIk0Uf\nV99Yz5LVS0iPTueGiTe4HY5SapBydIBbdd/TW55mR/kOHjn9EaJCo9wORyk1SGnPog8rqizi8U8e\n54ysMzh1+Kluh6OUGsQ0WfRRxhge+OgBRIR7prdb3V0ppXqNJos+6p097/B+0fvcnH8zQ6OHuh2O\nUmqQ02TRB1XVV/HARw8wNnEs3x7/bbfDUUopHeDui36z4TeUVJewbM4yQoL0V6SUcp/2LPqYz0o/\n47mtz7HghAVMSpnkdjhKKQVosuhTPMbDfR/eR3x4PLdNuc3tcJRSykuTRR/y4hcvsvHQRr5X8D3i\nw+PdDkcppbw0WfQRh44d4lfrf8X0odOZN2qe2+EopVQLmiz6iGWFy6hpqGHRzEWItHdGWqWUco8m\niz5gzf41vL7zda7Ou5qR8SPdDkcppdrQZOGyusY6lqxewvDY4Vx74rVuh6OUUu3SSfwuW75pObsq\ndvHbM39LREiE2+EopVS7tGfhoj0Ve/jdxt8xN3suszNmux2OUkp1SJOFS4wx3L/mfsKCw7hr2l1u\nh6OUUselycIlb+56kw+KP+DWybeSGpXqdjhKKXVcmixcUFlXyYNrH2TCkAksHLvQ7XCUUsovHeB2\nwa8//jWlNaU8csYjBAcFux2OUkr5pT2LXrb50GZWfLaChWMXkjsk1+1wlFIqIJoselGjp5GffPgT\nhkQO4dbJt7odjlJKBUyTRS9a8fkKtpZu5e5pdxMbFut2OEopFTAds+glJdUl/PrjXzMrfRbnZJ/j\ndjiqj6uvr6eoqIiamhq3Q1EDREREBJmZmYSGhnZpf00WveRna39GfWM9P5jxAy0UqPwqKioiNjaW\n7Oxs/XtR3WaM4fDhwxQVFTFyZNfqzzl6GEpE5orI5yKyXUTu6aDNN0Vki4hsFpE/+Wy/QkS22Zcr\nnIzTaf/e92/e3PUm1028jqy4LLfDUf1ATU0NQ4YM0USheoSIMGTIkG71VB3rWYhIMPAocBZQBKwV\nkZXGmC0+bXKAe4HZxpgyEUm1tycBPwYKAAOss/ctcypep9Q01HD/6vvJjsvm6ryr3Q5H9SOaKFRP\n6u7fk5M9i+nAdmPMTmNMHbACmN+qzXXAo01JwBhTYm8/B1hljCm171sFzHUwVsf87tPfUXS0iEUz\nFxEWHOZ2OEp12owZM8jPzycrK4uUlBTy8/PJz89n165dnXqcl19+mc8++6zTz3/yySezYcOGTu/X\n5Be/+AV/+tOf/Dd00Te+8Q127tzZ7n0lJSXMmTOH6Ohobr/99g4f4/Dhw5xxxhnk5ORwzjnnUF5e\n3qMxOpksMoC9PreL7G2+TgBOEJH/iMhqEZnbiX0RketFpFBECg8ePNiDofeMneU7Wb5pOfNGzWPG\nsBluh6NUl6xZs4YNGzawePFiFi5cyIYNG9iwYQPZ2dmdepyuJovuqK+v59lnn2Xhwr5dKeHGG2/k\n5z//ebv3RUVFcf/99/Pggw8e9zHuv/9+zj33XLZt28Ypp5zCz372sx6N0clk0V6fx7S6HQLkAHOA\nS4EnRSQhwH0xxjxhjCkwxhSkpKR0M9yeZYxhyeolRIZE8t2C77odjlKO+Nvf/sZJJ53ElClTWLhw\nIVVVVQDcddddTJgwgYkTJ3L33Xfzr3/9izfeeIM77rijS72SJn/84x858cQTycvL4/vf/753++OP\nP84JJ5zAnDlzuPbaa73fwFetWsW0adMIDrYqJaxevZqJEycya9Ys7rrrLvLz8wHYsWMHp5xyCpMn\nT2bq1KmsWbMGgLfeeovTTjuNBQsWkJOTw6JFi3jmmWeYNm0aEydO9L6Oyy67jJtvvpnTTjuN0aNH\n8/7773PFFVcwbtw4rrnmGm+c119/PQUFBeTm5rJ48WLv9jlz5vD3v/+dxsbGNq85JiaG2bNnExFx\n/FMYvPbaa1xxhTW8e8UVV/Dqq6929u09LidnQxUBw31uZwLF7bRZbYypB74Ukc+xkkcRVgLx3fc9\nxyJ1wOs7X2ftgbX8cOYPSY5Mdjsc1Y/95K+b2VJc0aOPOSE9jh9f0L0KAiUlJSxdupS3337b++33\n4Ycf5pprruGNN95g8+bNiAhHjhwhISGB8847jwULFnDRRRd16fmKiopYtGgRhYWFxMfHc+aZZ/L6\n668zadIkli5dyvr164mOjmbOnDlMnz4dgP/85z9MnTrV+xhXXXUVTz/9NNOnT+d73/ued/uwYcNY\ntWoVERERfPbZZ1xxxRXehPHJJ5+wdetW4uPjyc7O5qabbmLt2rUsW7aMRx55hF/84hcAlJeX8+67\n7/LSSy9xwQUX8OGHHzJu3DimTJnCpk2byMvLY+nSpSQlJdHQ0OBNQhMmTCA4OJjs7Gw2bdrEpEmT\nuvT+HD58mKYvzRkZGezfv79Lj9MRJ3sWa4EcERkpImHAJcDKVm1eBU4DEJFkrMNSO4E3gbNFJFFE\nEoGz7W39QnltOb8o/AUTUyay4IQFboejlCM++OADtmzZwqxZs8jPz+e5555j165dJCUlERQUxHXX\nXccrr7xCdHR0jzzfmjVrOP3000lOTiY0NJRvfetbvP/++97tiYmJhIWFsWBB8//c/v37vR+ghw4d\noq6uzptIvvWtb3nb1dbWcs0115CXl8cll1zCli3eeTjMmDGDtLQ0IiIiGDVqFOecY62TOvHEE1v0\nkC644ALv9vT0dCZMmEBQUBATJkzwtnv++eeZMmUKU6ZMYevWrS2eJzU1leLi1t+nu66nJ0g41rMw\nxjSIyC1YH/LBwHJjzGYRWQwUGmNW0pwUtgCNwF3GmMMAInIfVsIBWGyMKXUq1p728PqHKa8t54mz\nniBIdJG86p7u9gCcYoxh7ty5PPvss23uKywsZNWqVaxYsYLHHnuMf/zjHx0+ju8H+Ne//nV+9KMf\ndfh8ndkOEBkZ6Z0uerx2y5YtY/jw4fzxj3+kvr6emJgY733h4eHe60FBQd7bQUFBNDQ0tGnn28a3\n3bZt23j44Yf56KOPSEhI4LLLLmsxlbWmpobIyEhefPFFlixZAsBTTz3lPVTmz5AhQzh48CApKSns\n27ePoUOHBrRfoBz9JDPGvGGMOcEYM9oYc7+97Ud2osBY7jTGTDDGnGiMWeGz73JjzBj78gcn4+xJ\nG0o28Jcv/sK3x3+bsUlj3Q5HKcfMmjWLf/7zn95ZPFVVVWzbto3KykoqKiqYN28ev/zlL/n4448B\niI2NpbKyss3jhIWFeQfNO0oUADNnzuTdd9/l8OHDNDQ0sGLFCk499VRmzJjBu+++y5EjR6ivr+fl\nl1/27jN+/Hi2b98OQEpKCqGhoRQWFgKwYoX344by8nKGDRuGiPD0008fN7F0VUVFBbGxscTFxbF/\n/37efLPlwZJt27aRm5vLggULvO9HoIkC4MILL+Tpp58G4Omnn2b+/NaTT7tHV3D3oAZPA/etvo+0\nqDRuyr/J7XCUclRaWhq///3vWbhwIXV1dQD89Kc/JTIykq9//evU1tbi8Xh46KGHALj00ku54YYb\nWLZsGa+++mqnZ1NlZmayePFi5syZgzGGCy64gPPPPx+wBtSnT59ORkYGubm5xMfHA3Deeee1GGBe\nvnw5V111FbGxsXzta1/ztrvllltYsGABzz//PGeeeWaLnkFPmTJlChMmTCAvL49Ro0Yxe3bzqZSL\ni4uJj4+no4k6mZmZVFdXU19fz4svvsjbb7/N2LFjueqqq7jtttvIz8/n+9//Pt/85jd5/PHHGTly\nJC+88ELPvgBjzIC4TJ061bjtqU1Pmbyn8sxbu95yOxTVz23ZssXtEPqVyspKY4wxdXV15txzzzUr\nV6703nfBBReYHTt2tGhnjDFLliwxd955Z+8G2oGf/exn5qmnnnL8edr7u8IaFvD7GasH1HvIgaoD\nPLrhUU7NPJXTs053OxylBpUf/vCHTJ48mYkTJzJ27FjmzZvnve/BBx/0DhyvXLmS/Px88vLy+PDD\nD7n33nvdCrmFIUOGcNlll7kdxnGJceDYnBsKCgpM07FIN9z+7u38Z99/ePWiV8mIabN+UKlO2bp1\nK+PHj3c7DDXAtPd3JSLrjDEF/vbVnkUPeG/ve7y9521unHSjJgql1ICkyaKbquureWDNA4xJGMPl\nuZe7HY5SSjlCZ0N10+MbH6e4qpin5j5FaFDXTiqilFJ9nfYsumFb2Tae2fwMF4+5mKlpU/3voJRS\n/ZQmiy7yGA9LVi8hJiyGO6be4XY4SjlGS5Q7r3WJ8rVr15KXl8eYMWO44472P1+MMdx0002MGTOG\nSZMmdes9CoQmiy56bftrrC9Zz51T7yQxItHtcJRyjJYod17rEuU33ngjf/jDH9i2bRubN29m1apV\nbfb561//yt69e9m+fTuPPvooN998s6MxarLogrKaMpatW8aU1CnMH9OzS+qV6k+0RLn1OnqyRPne\nvXupqalh2rRpiAjf+c532i03/tprr3H55dakmpNPPpkDBw7g5Hl9dIC7Cx5a9xBVdVUsmrlICwUq\n5/3tHjjwac8+5tAT4dyl3XoILVHuTInyY8eOMXx489kdMjMz2bdvX5v3Y9++fe22c+rcPvpJ10mF\nBwp5dfurXJ57OTmJOW6Ho5RrtES5MyXK21so3V658UDb9RTtWXRCfWM9S1YvIT06nRsm3uB2OGqw\n6GYPwClGS5Q7UqI8MzOTvXubzypdVFREenp6m5ib2s2cOfO47XqK9iw64ektT7OjfAffn/F9okKj\n3A5HKVdpifLOCbRE+fDhwwkPD2ft2rUYY3j22WfbLTd+4YUX8swzzwDw73//m7S0NMcOQYH2LAJW\nVFnE4588zhlZZ3Dq8FPdDkcp12mJ8s7pTInyxx57jCuvvJKamhrmzZvHWWedBcCjjz5KeHg41157\nLRdccAF/+9vfGD16NNHR0d5zWTgmkNK0/eHiZIlyj8dj/nvVf5tpf5xm9h/d79jzKNVES5R3jpYo\nD4yWKHfY23ve5l/7/sXN+TczNLpnT1WolOo+LVHuPC1R7kdVfRUXvnohieGJrJi3gpAgPXKnnKcl\nypUTulOiXD/5/Hh0w6McrD7IQ3Me0kShlBq09DDUcWw9vJXntj7HghMWMCllktvhKKWUazRZdKDR\n08h9q+8jITyB26bc5nY4SinlKk0WHXhp20t8euhTvlfwPeLD490ORymlXKXJoh2Hjh3iV+t+xfSh\n05k3ap7/HZQawLREufNalyi/5557yMzMJCEh4bj7LVmyhDFjxjBu3DjeeustR2PUEdt2LCtcRk1j\nDYtmLnK01opS/UFTQb2nnnqKwsJCHnnkkS49zssvv0xQUBDjxo3ryfCOq6lE+fr163vtObuiqUT5\nY489BsD8+fO55ZZbyMvL63CfjRs38vLLL7Nlyxb27t3L3Llz+fzzzwkKcqYPoD2LVtbsX8PrO1/n\n6ryrGRk/0u1wlOrTtES59Tp6skQ5wEknncTQocdf0/Xaa69x6aWXEhYWxujRo8nKymLdunVdel8D\noT0LH3WNdSxZvYThscO59sRr3Q5HKQAe/OhBPivt2ZMGjUsax93T7+7WY2iJcmdKlE+aFNjMy337\n9jFnzhzv7aYS5dOmTevS++uP9ix8LN+0nF0Vu/jBjB8QERLhdjhK9WlaotyZEuWBam9Bdb8tUS4i\nc4GHgWDgSWPM0lb3Xwn8HGg6s8cjxpgn7fsagaYzvuwxxlzoZKx7Kvbwu42/Y272XGZnzPa/g1K9\npLs9AKcYLVHuSInyQAVayrynONazEJFg4FHgXGACcKmITGin6QvGmHz78qTP9mM+2x1NFMYY7l9z\nP2HBYdw17S4nn0qpAUNLlHdOoCXKA3XhhRfy/PPPU1dXx44dO9i9e3eLQ249zcnDUNOB7caYncaY\nOmAF0CdPWP3mrjf5oPgDbp18K6lRqW6Ho1S/4FuifNKkScyaNYsvvviC8vJyzj//fCZNmsTpp5/e\nokT5T3/60y4PcPuWKM/Pz2fmzJmcf/75ZGVleUuUn3322W1KlP/zn//0PkZTifJZs2YRFBTUokT5\nk08+ycyZM9m9e7fjJcqvu+6645Yov/POO8nOzqaiooLMzEyWLFkCwCuvvOIdGJ80aRIXXXQR48eP\n57zzzuM3v/mNYzOhAOdKlAMLsA49Nd3+DtZhJt82VwL7gY3Ai8Bwn/sagEJgNXBRB89xvd2mMCsr\nqysVe01FbYWZ88Ic882/ftM0NDZ06TGU6mlaorxztER5YPpqifL2Rlpa9+3+CmQbYyYCbwG+Z+/I\nMlYlxG8BvxKR0W0ezJgnjDEFxpiCrp4hqraxlhOTT+RHM39EcFBwlx5DKeUuLVHuPMdKlIvIScD/\nGmPOsW/fC2CMeaCD9sFAqTGmTW0NEXkKeN0Y82JHz+dUiXKl3KAlypUTulOi3MmexVogR0RGikgY\ncAmw0reBiAzzuXkhsNXenigi4fb1ZGA2sAWllFKucGzqrDGmQURuAd7Emjq73BizWUQWYx0jWwn8\nj4hciDU+UYo1hgEwHnhcRDxYCW2pMUaThRpUjDFabkb1mO4eRdIz5SnVB3355ZfExsYyZMgQTRiq\n24wxHD58mMrKSkaObFnGSM+Up1Q/lpmZSVFREQcPHnQ7FDVAREREkJmZ2eX9NVko1QeFhoa2+Qao\nlJu0NpRSSim/NFkopZTyS5OFUkopvwbMbCgROQjs7sZDJAOHeiicnqRxdY7G1TkaV+cMxLhGGGP8\nlsAYMMmiu0SkMJDpY71N4+ocjatzNK7OGcxx6WEopZRSfmmyUEop5Zcmi2ZPuB1ABzSuztG4Okfj\n6pxBG5eOWSillPJLexZKKaX8GrTJQkS+ISKbRcQjIh3OIhCRuSLyuYhsF5F7eiGuJBFZJSLb7J+J\nHbRrFJEN9mVle216KJ7jvn4RCReRF+z714hItlOxdCKmK0XkoM/7c63TMdnPu1xESkRkUwf3i4j8\nf3bcG0VkSh+Ja46IlPu8Xx2fCLtn4xouIu+KyFb7f/G2dtr0+nsWYFy9/p6JSISIfCQin9hx/aSd\nNs79PwZyOr2BeMEqgz4WeA8o6KBNMLADGAWEAZ8AExyO62fAPfb1e4AHO2h3tBfeI7+vH7gJ+K19\n/RLghT4Q05W0OoVvL/1NfQ2YAmzq4P7zgL9hnUVyJrCmj8Q1B+vkYr39fg0DptjXY4Ev2vld9vp7\nFmBcvf6e2e9BjH09FFgDzGzVxrH/x0HbszDGbDXGfO6n2XRguzFmpzGmDlgBzHc4tPk0n172aeAi\nh5/veAJ5/b7xvgicIc7W1HbjdxIQY8z7WOdl6ch84BljWQ0ktDoBmFtxucIYs98Ys96+Xol18rOM\nVs16/T0LMK5eZ78HR+2bofal9aCzY/+PgzZZBCgD2Otzuwjn/2jSjDH7wfqjBVI7aBchIoUislpE\nnEoogbx+bxtjTANQDgxxKJ5AYwL4L/uwxYsiMtzBeDrDjb+nQJ1kH974m4jk9vaT24dLJmN9W/bl\n6nt2nLjAhfdMRIJFZANQAqwyxnT4fvX0/+OALlEuIm8BQ9u56wfGmNcCeYh2tnV7+tjx4urEw2QZ\nY4pFZBTwjoh8aozZ0d3YWgnk9TvyHh1HIM/3V+B5Y0ytiNyI9U3rdAdjClRvv1eBWo9V8uGoiJwH\nvArk9NaTi0gM8BJwuzGmovXd7ezSK++Zn7hcec+MMY1AvogkAK+ISJ4xxncsyrH3a0AnC2PMmd18\niCLA91tpJlDczcc8blwi8pWIDDPG7Le72yUdPEax/XOniLyH9e2np5NFIK+/qU2RiIQA8Th7yMNv\nTMaYwz43fwc86GA8neHI31N3+X4QGmPeEJHfiEiyMcbxGkgiEor1gfycMebldpq48p75i8vN98x+\nziP2//1cwDdZOPb/qIehjm8tkCMiI0UkDGvAyLGZR7aVwBX29SuANj0gEUkUkXD7ejIwG3DiHOWB\nvH7feBcA7xh7dM0hfmNqdUz7Qqxjzn3BSuBye4bPTKC86ZCjm0RkaNNxbRGZjvW5cPj4e/XI8wrw\ne2CrMeahDpr1+nsWSFxuvGcikmL3KBCRSOBM4LNWzZz7f+zN0fy+dAEuxsrCtcBXwJv29nTgDZ92\n52HNhtiBdfjK6biGAG8D2+yfSfb2AuBJ+/os4FOsmUCfAtc4GE+b1w8sBi60r0cAfwG2Ax8Bo3rh\nPfIX0wPAZvv9eRcY10t/U88D+4F6+2/rGuBG4Eb7fgEeteP+lA5m4bkQ1y0+79dqYFYvxXUy1iGS\njcAG+3Ke2+9ZgHH1+nsGTAQ+tuPaBPzI3t4r/4+6glsppZRfehhKKaWUX5oslFJK+aXJQimllF+a\nLJRSSvmlyUIppZRfmiyU6gQROeq/1XH3f9FedY+IxIjI4yKyw64i+r6IzBCRMPv6gF40q/oXTRZK\n9RK7flCwMWanvelJrNW1OcaYXKxqucnGKpD4NrDQlUCVaocmC6W6wF5R/HMR2SQin4rIQnt7kF36\nYbOIvC4ib4jIAnu3b2OvyBeR0cAMYJExxgNW6RZjzP/ZbV+12yvVJ2g3V6mu+TqQD0wCkoG1IvI+\nVumVbOBErIrBW4Hl9j6zsVZTA+QCG4xVGK49m4BpjkSuVBdoz0KprjkZq7JtozHmK+CfWB/uJwN/\nMcZ4jDEHsMqNNBkGHAzkwe0kUicisT0ct1JdoslCqa7p6IQyxzvRzDGs2j1g1RWaJCLH+x8MB2q6\nEJtSPU6ThVJd8z6w0D4ZTQrWqUs/Av6NdeKlIBFJwzr9ZpOtwBgAY517pBD4iU/10hwRmW9fHwIc\nNMbU99YLUup4NFko1TWvYFX//AR4B/h/9mGnl7Aqu24CHsc6w1q5vc//0TJ5XIt1EqztIvIp1rk3\nms7VcBrwhrMvQanAadVZpXqYiMQY6wxqQ7B6G7ONMQfscxC8a9/uaGC76TFeBu41/s8Tr1Sv0NlQ\nSvW81+2T1IQB99k9Dowxx0Tkx1jnSd7T0c72SZ1e1USh+hLtWSillPJLxyyUUkr5pclCKaWUX5os\nlFJK+aXJQimllF+aLJRSSvmlyUIppZRf/z9HSj2AamWDPQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10f944890>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#accuracy_s1 =np.array(accuracy_s).reshape(len(C_s),len(gamma_s))\n",
    "Otto_SVM_result = pd.read_csv(\"Otto_SVM_result.csv\")\n",
    "accuracy_s1 = Otto_SVM_result['accuracy']\n",
    "\n",
    "C_s = np.logspace(-1, 3, 5)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份 \n",
    "gamma_s = np.logspace(-1, 1, 3)  \n",
    "accuracy_s1 =np.array(accuracy_s1).reshape(len(C_s),len(gamma_s))\n",
    "\n",
    "x_axis = np.log10(C_s)\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    plt.plot(x_axis, np.array(accuracy_s1[:,j]), label = ' Test - log(gamma)' + str(np.log10(gamma)))\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'accuracy' )\n",
    "plt.savefig('RBF_SVM_Otto.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10.0\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "### 最佳超参数\n",
    "index = np.unravel_index(np.argmax(accuracy_s1, axis=None), accuracy_s1.shape)\n",
    "Best_C = C_s[ index[0] ]\n",
    "Best_gamma = gamma_s[ index[1] ]\n",
    "\n",
    "print(Best_C)\n",
    "print(Best_gamma)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 找到最佳参数后，用全体训练数据训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# SVC训练SVC，支持概率输出\n",
    "Best_C = 100\n",
    "Best_gamma = 1.0\n",
    "\n",
    "SVC4 =  SVC( C = Best_C, kernel='rbf', gamma = Best_gamma, probability=True)\n",
    "SVC4.fit(X_train, y_train)\n",
    "\n",
    "#保持模型，用于后续测试\n",
    "import cPickle\n",
    "cPickle.dump(SVC4, open(\"Otto_RBF_SVC.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
